{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1.导入工具包 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import division\n",
    "\n",
    "import pickle\n",
    "import numpy as np\n",
    "import scipy.io as sio\n",
    "import scipy.sparse as ss\n",
    "from numpy.random import random  \n",
    "from collections import defaultdict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2.RecommonderSystem类 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "dpath = \"./data/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 将所有特征串联起来，构成RS_Train.csv\n",
    "#RS_Test.csv\n",
    "#为最后推荐系统做准备\n",
    "class RecommonderSystem:\n",
    "    def __init__(self):\n",
    "        # 读入数据做初始化\n",
    "    \n",
    "        #用户和活动新的索引\n",
    "        self.userIndex = pickle.load(open(\"PE_userIndex.pkl\", 'rb'))\n",
    "        self.eventIndex = pickle.load(open(\"PE_eventIndex.pkl\", 'rb'))\n",
    "        self.n_users = len(self.userIndex)\n",
    "        self.n_items = len(self.eventIndex)\n",
    "    \n",
    "        #用户-活动关系矩阵R\n",
    "        #在train_SVD会重新从文件中读取,二者要求的格式不同，来不及统一了:(\n",
    "        self.userEventScores = sio.mmread(\"PE_userEventScores\").todense()\n",
    "    \n",
    "        #倒排表\n",
    "        ##每个用户参加的事件\n",
    "        self.itemsForUser = pickle.load(open(\"PE_eventsForUser.pkl\", 'rb'))\n",
    "        ##事件参加的用户\n",
    "        self.usersForItem = pickle.load(open(\"PE_usersForEvent.pkl\", 'rb'))\n",
    "    \n",
    "        #基于模型的协同过滤参数初始化,训练\n",
    "        self.init_SVD()\n",
    "        self.train_SVD(trainfile = dpath+\"train.csv\")\n",
    "    \n",
    "        #根据用户属性计算出的用户之间的相似度\n",
    "        self.userSimMatrix = sio.mmread(\"US_userSimMatrix\").todense()\n",
    "    \n",
    "        #根据活动属性计算出的活动之间的相似度\n",
    "        self.eventPropSim = sio.mmread(\"EV_eventPropSim\").todense()\n",
    "        self.eventContSim = sio.mmread(\"EV_eventContSim\").todense()\n",
    "    \n",
    "        #每个用户的朋友的数目\n",
    "        self.numFriends = sio.mmread(\"UF_numFriends\")\n",
    "        #用户的每个朋友参加活动的分数对该用户的影响\n",
    "        self.userFriends = sio.mmread(\"UF_userFriends\").todense()\n",
    "    \n",
    "        #活动本身的热度\n",
    "        self.eventPopularity = sio.mmread(\"EA_eventPopularity\").todense()\n",
    "\n",
    "    def init_SVD(self, K=20):\n",
    "        #初始化模型参数（for 基于模型的协同过滤SVD_CF）\n",
    "        self.K = K  \n",
    "    \n",
    "        #init parameters\n",
    "        #bias\n",
    "        self.bi = np.zeros(self.n_items)  \n",
    "        self.bu = np.zeros(self.n_users)  \n",
    "    \n",
    "        #the small matrix\n",
    "        self.P = random((self.n_users,self.K))/10*(np.sqrt(self.K))\n",
    "        self.Q = random((self.K, self.n_items))/10*(np.sqrt(self.K))\n",
    "          \n",
    "    def train_SVD(self,trainfile = dpath+'train.csv', steps=100,gamma=0.04,Lambda=0.15):\n",
    "        #训练SVD模型（for 基于模型的协同过滤SVD_CF）\n",
    "        #gamma：为学习率\n",
    "        #Lambda：正则参数\n",
    "    \n",
    "        #偷懒了，为了和原来的代码的输入接口一样，直接从训练文件中去读取数据\n",
    "        print (\"SVD Train...\")\n",
    "        ftrain = open(trainfile, 'r')\n",
    "        ftrain.readline()\n",
    "        self.mu = 0.0\n",
    "        n_records = 0\n",
    "        uids = []  #每条记录的用户索引\n",
    "        i_ids = [] #每条记录的item索引\n",
    "        #用户-Item关系矩阵R（内容同userEventScores相同），临时变量，训练完了R不再需要\n",
    "        R = np.zeros((self.n_users, self.n_items))\n",
    "    \n",
    "        for line in ftrain:\n",
    "            cols = line.strip().split(\",\")\n",
    "            u = self.userIndex[cols[0]]  #用户\n",
    "            i = self.eventIndex[cols[1]] #活动\n",
    "        \n",
    "            uids.append(u)\n",
    "            i_ids.append(i)\n",
    "        \n",
    "            R[u,i] = int(cols[4])  #interested\n",
    "            self.mu += R[u,i]\n",
    "            n_records += 1\n",
    "    \n",
    "        ftrain.close()\n",
    "        self.mu /= n_records\n",
    "    \n",
    "        # 请补充完整SVD模型训练过程\n",
    "        print (\"SVD training started\")\n",
    "        for step in range(steps):# 最大重复次数为steps\n",
    "            rmse = 0\n",
    "            mse = 0\n",
    "            \n",
    "            for j in range(n_records):\n",
    "                u = uids[j]\n",
    "                i = i_ids[j]\n",
    "                \n",
    "                #残差\n",
    "                #Rui_hat = self.mu + (P[u,:] * Q[:,i]) + bu[u] + bi[i]\n",
    "                eui = R[u,i] - self.pred_SVD(u,i)\n",
    "                \n",
    "                #残差平方和\n",
    "                mse += eui ** 2\n",
    "                \n",
    "                #梯度下降\n",
    "                #更新偏差项\n",
    "                self.bu[u] += gamma * (eui - Lambda * self.bu[u])\n",
    "                self.bi[i] += gamma * (eui - Lambda * self.bi[i])\n",
    "                #更新分解矩阵\n",
    "                for k in range(self.K):\n",
    "                    self.P[u,k] += gamma * (eui * self.Q[k,i]) - Lambda * self.P[u,k]\n",
    "                    self.Q[k,i] += gamma * (eui * self.P[u,k]) - Lambda * self.Q[k,i]\n",
    "                \n",
    "            rmse = np.sqrt(mse)\n",
    "            print(\"step: \", step, \"rmse:\", rmse)\n",
    "        print (\"SVD trained\")\n",
    "    \n",
    "    def pred_SVD(self, uid, i_id):\n",
    "        #根据当前参数，预测用户uid对Item（i_id）的打分        \n",
    "        ans=self.mu + self.bi[i_id] + self.bu[uid] + np.dot(self.P[uid,:],self.Q[:,i_id])  \n",
    "        \n",
    "        #将打分范围控制在0-1之间\n",
    "        if ans>1:  \n",
    "            return 1  \n",
    "        elif ans<0:  \n",
    "            return 0\n",
    "        return ans  \n",
    "\n",
    "    def sim_cal_UserCF(self, uid1, uid2 ):\n",
    "        #请补充基于用户的协同过滤中的两个用户uid1和uid2之间的相似度（根据两个用户对item打分的相似度）\n",
    "        similarity = 0.0\n",
    "        #uid1和uid2打过分的item\n",
    "        items_u1 = self.itemsForUser[uid1]\n",
    "        items_u2 = self.itemsForUser[uid2]\n",
    "        \n",
    "        #uid1和uid2对各item的打分\n",
    "        score_u1 = self.userEventScores[uid1]\n",
    "        score_u2 = self.userEventScores[uid2]\n",
    "        \n",
    "        #uid1和uid2对common item的打分\n",
    "        common_item_score_u1 = []\n",
    "        common_item_score_u2 = []\n",
    "        \n",
    "        #两者都打过分的item\n",
    "        common_items = {}\n",
    "        for item in items_u1:\n",
    "            if item in items_u2:\n",
    "                common_items[item] = 1\n",
    "                common_item_score_u1.append(score_u1[0,item])\n",
    "                common_item_score_u2.append(score_u2[0,item])\n",
    "        \n",
    "        #没有两者都打过分的item，直接返回0\n",
    "        if len(common_items) == 0:\n",
    "            return 0,0,0\n",
    "        #均值\n",
    "        mean_u1 = np.mean(common_item_score_u1)\n",
    "        mean_u2 = np.mean(common_item_score_u2)\n",
    "        \n",
    "        #评分减去均值\n",
    "        common_item_score_u1 = common_item_score_u1 - mean_u1\n",
    "        common_item_score_u2 = common_item_score_u2 - mean_u2\n",
    "        \n",
    "        #求和\n",
    "        sum_sq_u1 = np.sum(common_item_score_u1 ** 2)\n",
    "        sum_sq_u2 = np.sum(common_item_score_u2 ** 2)\n",
    "        \n",
    "        #分子\n",
    "        numerator = np.sum(common_item_score_u1 * common_item_score_u2)\n",
    "        #分母\n",
    "        denominator = np.sqrt(sum_sq_u1 * sum_sq_u2)\n",
    "        #分母为0时，直接返回0\n",
    "        if denominator == 0:\n",
    "            return 0,0,0\n",
    "        else:\n",
    "            similarity = numerator / denominator\n",
    "            \n",
    "        return similarity,mean_u1,mean_u2\n",
    "\n",
    "    def userCFReco(self, userId, eventId):\n",
    "        \"\"\"\n",
    "        根据User-based协同过滤，得到event的推荐度\n",
    "        基本的伪代码思路如下：\n",
    "        for item i\n",
    "          for every other user v that has a preference for i\n",
    "            compute similarity s between u and v\n",
    "            incorporate v's preference for i weighted by s into running average\n",
    "        return top items ranked by weighted average\n",
    "        \"\"\"\n",
    "        #请补充完整代码\n",
    "        ans = 0.0\n",
    "        sim_accumulate = 0.0\n",
    "        preference = 0.0\n",
    "        \n",
    "        uid1 = self.userIndex[userId]\n",
    "        item = self.eventIndex[eventId]\n",
    "        \n",
    "        #用户的平均评分，不太清楚是不是需要，看ppt的公式是需要计算的\n",
    "        mean_u1 = 0.0\n",
    "        mean_u2 = 0.0\n",
    "        \n",
    "        #对该item有评分的用户\n",
    "        uids = self.usersForItem[item]\n",
    "        \n",
    "        for uid2 in uids:\n",
    "            similarity, mean_u1, mean_u2 = self.sim_cal_UserCF(uid1, uid2)\n",
    "            sim_accumulate += similarity\n",
    "            preference += similarity * (self.userEventScores[uid2, item] - mean_u2)\n",
    "            \n",
    "        #相似度和为0时，直接返回0\n",
    "        if sim_accumulate == 0:\n",
    "            return 0\n",
    "        \n",
    "        ans = mean_u1 + (preference/sim_accumulate)\n",
    "        return ans\n",
    "\n",
    "\n",
    "    def sim_cal_ItemCF(self, i_id1, i_id2):\n",
    "        #计算Item i_id1和i_id2之间的相似性\n",
    "        #请补充完整代码\n",
    "        similarity = 0.0\n",
    "        \n",
    "        #对item1和item2打过分的用户\n",
    "        users_i1 = self.usersForItem[i_id1]\n",
    "        users_i2 = self.usersForItem[i_id2]\n",
    "        \n",
    "        #各用户对item1和item2的打分情况\n",
    "        score_i1 = self.userEventScores[:,i_id1]\n",
    "        score_i2 = self.userEventScores[:,i_id2]\n",
    "        \n",
    "        #common user对item1和item2的打分\n",
    "        common_user_score_i1 = []\n",
    "        common_user_score_i2 = []\n",
    "        \n",
    "        #两个item都打过分的用户\n",
    "        common_users = {}\n",
    "        for user in users_i1:\n",
    "            if user in users_i2:\n",
    "                common_users[user] = 1\n",
    "                user_mean = np.mean(self.userEventScores[user])#均值\n",
    "                i1 = score_i1[user,0] - user_mean\n",
    "                i2 = score_i2[user,0] - user_mean\n",
    "                common_user_score_i1.append(i1)\n",
    "                common_user_score_i2.append(i2)\n",
    "                \n",
    "        #没有两者都打过分的item，直接返回0\n",
    "        if len(common_users) == 0:\n",
    "            return 0\n",
    "        \n",
    "        #求和\n",
    "        #直接求和总报错：TypeError: unsupported operand type(s) for ** or pow(): 'list' and 'int'，同样的代码基于用户那块就没问题。。\n",
    "        #sum_sq_i1 = np.sum(common_user_score_i1 ** 2)\n",
    "        #sum_sq_i2 = np.sum(common_user_score_i2 ** 2)\n",
    "        sum_sq_i1 = np.sum(user_score1 ** 2 for user_score1 in common_user_score_i1)\n",
    "        sum_sq_i2 = np.sum(user_score2 ** 2 for user_score2 in common_user_score_i2)\n",
    "        \n",
    "        #分子\n",
    "        #numerator = np.sum(common_user_score_i1 * common_user_score_i2)\n",
    "        numerator = np.sum(common_user_score_i1[i] * common_user_score_i2[i] for i in range(len(common_user_score_i1)))\n",
    "        \n",
    "        #分母\n",
    "        denominator = np.sqrt(sum_sq_i1 * sum_sq_i2)\n",
    "        \n",
    "        #分母为0时，直接返回0\n",
    "        if denominator == 0:\n",
    "            return 0\n",
    "        else:\n",
    "            similarity = numerator / denominator\n",
    "            \n",
    "        return similarity\n",
    "    \n",
    "    def eventCFReco(self, userId, eventId):    \n",
    "        \"\"\"\n",
    "        根据基于物品的协同过滤，得到Event的推荐度\n",
    "        基本的伪代码思路如下：\n",
    "        for item i \n",
    "            for every item j tht u has a preference for\n",
    "                compute similarity s between i and j\n",
    "                add u's preference for j weighted by s to a running average\n",
    "        return top items, ranked by weighted average\n",
    "        \"\"\"\n",
    "        #请补充完整代码\n",
    "        ans = 0.0\n",
    "        sim_accumulate = 0.0\n",
    "        preference = 0.0\n",
    "        \n",
    "        uid = self.userIndex[userId]\n",
    "        i_id1 = self.eventIndex[eventId]\n",
    "        \n",
    "        #该用户有评分的物品\n",
    "        items = self.itemsForUser[i_id1]\n",
    "        \n",
    "        for i_id2 in items:\n",
    "            similarity = self.sim_cal_ItemCF(i_id1, i_id2)\n",
    "            sim_accumulate += similarity\n",
    "            preference += similarity * (self.userEventScores[uid, i_id2])\n",
    "            \n",
    "        #相似度和为0时，直接返回0\n",
    "        if sim_accumulate == 0:\n",
    "            return 0\n",
    "        \n",
    "        ans = preference/sim_accumulate\n",
    "        return ans\n",
    "    \n",
    "    def svdCFReco(self, userId, eventId):\n",
    "        #基于模型的协同过滤, SVD++/LFM\n",
    "        u = self.userIndex[userId]\n",
    "        i = self.eventIndex[eventId]\n",
    "\n",
    "        return self.pred_SVD(u,i)\n",
    "\n",
    "    def userReco(self, userId, eventId):\n",
    "        \"\"\"\n",
    "        类似基于User-based协同过滤，只是用户之间的相似度由用户本身的属性得到，计算event的推荐度\n",
    "        基本的伪代码思路如下：\n",
    "        for item i\n",
    "          for every other user v that has a preference for i\n",
    "            compute similarity s between u and v\n",
    "            incorporate v's preference for i weighted by s into running aversge\n",
    "        return top items ranked by weighted average\n",
    "        \"\"\"\n",
    "        i = self.userIndex[userId]\n",
    "        j = self.eventIndex[eventId]\n",
    "\n",
    "        vs = self.userEventScores[:, j]\n",
    "        sims = self.userSimMatrix[i, :]\n",
    "\n",
    "        prod = sims * vs\n",
    "\n",
    "        try:\n",
    "            return prod[0, 0] - self.userEventScores[i, j]\n",
    "        except IndexError:\n",
    "            return 0\n",
    "\n",
    "    def eventReco(self, userId, eventId):\n",
    "        \"\"\"\n",
    "        类似基于Item-based协同过滤，只是item之间的相似度由item本身的属性得到，计算Event的推荐度\n",
    "        基本的伪代码思路如下：\n",
    "        for item i \n",
    "          for every item j that u has a preference for\n",
    "            compute similarity s between i and j\n",
    "            add u's preference for j weighted by s to a running average\n",
    "        return top items, ranked by weighted average\n",
    "        \"\"\"\n",
    "        i = self.userIndex[userId]\n",
    "        j = self.eventIndex[eventId]\n",
    "        js = self.userEventScores[i, :]\n",
    "        psim = self.eventPropSim[:, j]\n",
    "        csim = self.eventContSim[:, j]\n",
    "        pprod = js * psim\n",
    "        cprod = js * csim\n",
    "    \n",
    "        pscore = 0\n",
    "        cscore = 0\n",
    "        try:\n",
    "            pscore = pprod[0, 0] - self.userEventScores[i, j]\n",
    "        except IndexError:\n",
    "            pass\n",
    "        try:\n",
    "            cscore = cprod[0, 0] - self.userEventScores[i, j]\n",
    "        except IndexError:\n",
    "            pass\n",
    "        return pscore, cscore\n",
    "\n",
    "    def userPop(self, userId):\n",
    "        \"\"\"\n",
    "        基于用户的朋友个数来推断用户的社交程度\n",
    "        主要的考量是如果用户的朋友非常多，可能会更倾向于参加各种社交活动\n",
    "        \"\"\"\n",
    "        if userId in self.userIndex:\n",
    "            i = self.userIndex[userId]\n",
    "            try:\n",
    "                return self.numFriends[0, i]\n",
    "            except IndexError:\n",
    "                return 0\n",
    "        else:\n",
    "            return 0\n",
    "\n",
    "    def friendInfluence(self, userId):\n",
    "        \"\"\"\n",
    "        朋友对用户的影响\n",
    "        主要考虑用户所有的朋友中，有多少是非常喜欢参加各种社交活动/event的\n",
    "        用户的朋友圈如果都积极参与各种event，可能会对当前用户有一定的影响\n",
    "        \"\"\"\n",
    "        nusers = np.shape(self.userFriends)[1]\n",
    "        i = self.userIndex[userId]\n",
    "        return (self.userFriends[i, :].sum(axis=0) / nusers)[0,0]\n",
    "\n",
    "    def eventPop(self, eventId):\n",
    "        \"\"\"\n",
    "        本活动本身的热度\n",
    "        主要是通过参与的人数来界定的\n",
    "        \"\"\"\n",
    "        i = self.eventIndex[eventId]\n",
    "        return self.eventPopularity[i, 0]\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3.测试RecommendSystem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SVD Train...\n",
      "SVD training started\n",
      "step:  0 rmse: 75.8301489730892\n",
      "step:  1 rmse: 51.03009761536536\n",
      "step:  2 rmse: 48.42894825759943\n",
      "step:  3 rmse: 46.680493515682215\n",
      "step:  4 rmse: 45.21534484015678\n",
      "step:  5 rmse: 43.974512183824466\n",
      "step:  6 rmse: 42.89951570939152\n",
      "step:  7 rmse: 41.947930851201015\n",
      "step:  8 rmse: 41.092429982124045\n",
      "step:  9 rmse: 40.31517364758756\n",
      "step:  10 rmse: 39.60403444817289\n",
      "step:  11 rmse: 38.95041453612901\n",
      "step:  12 rmse: 38.34776648822599\n",
      "step:  13 rmse: 37.79102032613358\n",
      "step:  14 rmse: 37.27585213950168\n",
      "step:  15 rmse: 36.79854669246539\n",
      "step:  16 rmse: 36.3558431302084\n",
      "step:  17 rmse: 35.94483483223498\n",
      "step:  18 rmse: 35.56297060183848\n",
      "step:  19 rmse: 35.20794259776389\n",
      "step:  20 rmse: 34.877634550558106\n",
      "step:  21 rmse: 34.570133826748865\n",
      "step:  22 rmse: 34.28368623036478\n",
      "step:  23 rmse: 34.0166745494121\n",
      "step:  24 rmse: 33.76763078648688\n",
      "step:  25 rmse: 33.53520486713976\n",
      "step:  26 rmse: 33.318145874772995\n",
      "step:  27 rmse: 33.11531696549248\n",
      "step:  28 rmse: 32.92565929854393\n",
      "step:  29 rmse: 32.74819945642969\n",
      "step:  30 rmse: 32.582040082150264\n",
      "step:  31 rmse: 32.42636128234072\n",
      "step:  32 rmse: 32.28041509402401\n",
      "step:  33 rmse: 32.14349925789371\n",
      "step:  34 rmse: 32.01497152491508\n",
      "step:  35 rmse: 31.8942323860349\n",
      "step:  36 rmse: 31.780724657076522\n",
      "step:  37 rmse: 31.67395035229531\n",
      "step:  38 rmse: 31.57343984706669\n",
      "step:  39 rmse: 31.478760596101342\n",
      "step:  40 rmse: 31.389515288199398\n",
      "step:  41 rmse: 31.305332959735576\n",
      "step:  42 rmse: 31.225870315649907\n",
      "step:  43 rmse: 31.150808621294033\n",
      "step:  44 rmse: 31.07985846064134\n",
      "step:  45 rmse: 31.012747573513213\n",
      "step:  46 rmse: 30.94922533469163\n",
      "step:  47 rmse: 30.88905995208262\n",
      "step:  48 rmse: 30.832037853195743\n",
      "step:  49 rmse: 30.77796006095567\n",
      "step:  50 rmse: 30.726639714642413\n",
      "step:  51 rmse: 30.677904983680264\n",
      "step:  52 rmse: 30.63159590596619\n",
      "step:  53 rmse: 30.58756446222138\n",
      "step:  54 rmse: 30.54567338083027\n",
      "step:  55 rmse: 30.505793031975717\n",
      "step:  56 rmse: 30.467804248707193\n",
      "step:  57 rmse: 30.43159332731148\n",
      "step:  58 rmse: 30.39705827756485\n",
      "step:  59 rmse: 30.36410277157099\n",
      "step:  60 rmse: 30.332637530844302\n",
      "step:  61 rmse: 30.302577894859876\n",
      "step:  62 rmse: 30.273844337490356\n",
      "step:  63 rmse: 30.246365119442025\n",
      "step:  64 rmse: 30.220072639488098\n",
      "step:  65 rmse: 30.194902620940898\n",
      "step:  66 rmse: 30.170794458272685\n",
      "step:  67 rmse: 30.147677254452333\n",
      "step:  68 rmse: 30.125515085157478\n",
      "step:  69 rmse: 30.104259053683396\n",
      "step:  70 rmse: 30.083863072249663\n",
      "step:  71 rmse: 30.064284882650046\n",
      "step:  72 rmse: 30.04548280096249\n",
      "step:  73 rmse: 30.027418953142373\n",
      "step:  74 rmse: 30.010057473713427\n",
      "step:  75 rmse: 29.993364201863304\n",
      "step:  76 rmse: 29.97730740759493\n",
      "step:  77 rmse: 29.961856586758724\n",
      "step:  78 rmse: 29.946983351894318\n",
      "step:  79 rmse: 29.93266093331313\n",
      "step:  80 rmse: 29.918864135618033\n",
      "step:  81 rmse: 29.905569359251402\n",
      "step:  82 rmse: 29.892754109779936\n",
      "step:  83 rmse: 29.8803970291142\n",
      "step:  84 rmse: 29.86847794369247\n",
      "step:  85 rmse: 29.856977135589187\n",
      "step:  86 rmse: 29.8458764723141\n",
      "step:  87 rmse: 29.835158823340883\n",
      "step:  88 rmse: 29.824807835333022\n",
      "step:  89 rmse: 29.814808014679382\n",
      "step:  90 rmse: 29.805144819987706\n",
      "step:  91 rmse: 29.79580415222236\n",
      "step:  92 rmse: 29.786772864031125\n",
      "step:  93 rmse: 29.778038468515962\n",
      "step:  94 rmse: 29.769589110118584\n",
      "step:  95 rmse: 29.761413392241383\n",
      "step:  96 rmse: 29.753500381103063\n",
      "step:  97 rmse: 29.74583993480607\n",
      "step:  98 rmse: 29.738422013767742\n",
      "step:  99 rmse: 29.73123695106873\n",
      "SVD trained\n"
     ]
    }
   ],
   "source": [
    "#构造RS实例\n",
    "RS = RecommonderSystem()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.5773502691896258, 0.25, 0.5)"
      ]
     },
     "execution_count": 146,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#测试基于用户的协同过滤的相似度计算\n",
    "RS.sim_cal_UserCF(1,3203)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.1"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#测试基于用户的协同过滤获得推荐度\n",
    "RS.userCFReco(\"1528359455\",\"1390707377\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 + 91 :  1.0\n",
      "0 + 206 :  -1.0\n",
      "0 + 365 :  -0.0001863321612679168\n",
      "4 + 211 :  1.0\n",
      "6 + 365 :  -0.9999999999999998\n",
      "7 + 121 :  1.0\n",
      "7 + 184 :  1.0\n",
      "7 + 453 :  1.0\n",
      "8 + 403 :  1.0\n",
      "11 + 151 :  -0.9999999999999998\n",
      "16 + 194 :  -1.0\n",
      "16 + 435 :  1.0\n",
      "21 + 415 :  1.0\n",
      "22 + 259 :  -1.0\n",
      "27 + 90 :  -0.9999999999999999\n",
      "31 + 332 :  -1.0\n",
      "37 + 370 :  -1.0\n",
      "48 + 183 :  1.0\n",
      "58 + 219 :  -1.0\n",
      "61 + 264 :  1.0\n",
      "62 + 213 :  1.0\n",
      "64 + 483 :  1.0\n",
      "79 + 148 :  -0.9999999999999998\n",
      "84 + 290 :  1.0\n",
      "84 + 348 :  1.0\n",
      "84 + 424 :  -0.9999999999999998\n",
      "85 + 276 :  -1.0\n",
      "88 + 368 :  -1.0\n",
      "91 + 365 :  -0.9999999999999998\n",
      "103 + 388 :  -0.9999999999999998\n",
      "114 + 270 :  1.0\n",
      "119 + 208 :  1.0\n",
      "121 + 184 :  1.0\n",
      "121 + 453 :  1.0\n",
      "122 + 166 :  -0.9999999999999998\n",
      "124 + 171 :  1.0\n",
      "132 + 249 :  -1.0\n",
      "136 + 323 :  1.0\n",
      "143 + 380 :  -1.0\n",
      "163 + 359 :  -1.0\n",
      "163 + 476 :  -1.0\n",
      "175 + 187 :  -0.9999999999999998\n",
      "179 + 324 :  1.0\n",
      "184 + 453 :  1.0\n",
      "189 + 438 :  -0.9999999999999998\n",
      "189 + 495 :  -0.9999999999999998\n",
      "194 + 435 :  -1.0\n",
      "195 + 362 :  1.0\n",
      "203 + 227 :  1.0\n",
      "204 + 378 :  1.0\n",
      "221 + 398 :  -1.0\n",
      "227 + 343 :  1.0\n",
      "227 + 499 :  1.0\n",
      "228 + 408 :  -0.9999999999999998\n",
      "229 + 321 :  1.0\n",
      "230 + 295 :  1.0\n",
      "230 + 310 :  1.0\n",
      "230 + 468 :  1.0\n",
      "232 + 289 :  1.0\n",
      "234 + 433 :  1.0\n",
      "246 + 292 :  -0.9999999999999998\n",
      "253 + 492 :  1.0\n",
      "257 + 345 :  -0.9999999999999999\n",
      "257 + 365 :  -1.0\n",
      "257 + 424 :  1.0\n",
      "263 + 450 :  1.0\n",
      "264 + 338 :  1.0\n",
      "268 + 410 :  1.0\n",
      "271 + 306 :  1.0\n",
      "274 + 368 :  1.0\n",
      "295 + 310 :  1.0\n",
      "295 + 468 :  1.0\n",
      "310 + 468 :  1.0\n",
      "343 + 499 :  1.0\n",
      "353 + 400 :  1.0\n",
      "359 + 476 :  1.0\n",
      "364 + 392 :  1.0\n",
      "364 + 449 :  -0.9999999999999999\n",
      "365 + 487 :  1.0\n",
      "392 + 449 :  -0.9999999999999999\n",
      "438 + 495 :  1.0\n",
      "461 + 464 :  1.0\n",
      "472 + 477 :  1.0\n"
     ]
    }
   ],
   "source": [
    "#测试基于物品的协同过滤的相似度计算\n",
    "#试了好几组都是0，没法验证是不是有问题，干脆把不是0的都打印一下看看。。。\n",
    "for i in range(500):\n",
    "    for j in range(i, 500):\n",
    "        if i != j and RS.sim_cal_ItemCF(i,j) != 0:\n",
    "            print(i,\"+\",j,\": \",RS.sim_cal_ItemCF(i,j))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "数值有些奇怪。。查了很久。。不知道是不是哪里写错了"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7068695073668977"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#测试基于物品的协同过滤的相似度计算\n",
    "RS.sim_cal_ItemCF(0, 5118)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "460925649\n",
      "3584466415\n",
      "1611840162\n",
      "2645454646\n"
     ]
    }
   ],
   "source": [
    "#找到基于物品的协同过滤推荐度不为0的item（用户3044012）\n",
    "for i in RS.eventIndex:\n",
    "    if RS.eventCFReco(\"3044012\", i) != 0:\n",
    "        print(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#测试基于物品的协同过滤获得推荐度\n",
    "RS.eventCFReco(\"3044012\", \"460925649\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4.generateRSData方法"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generateRSData(RS, train=True, header=True):\n",
    "    \"\"\"\n",
    "    把前面user-based协同过滤 和 item-based协同过滤，以及各种热度和影响度作为特征组合在一起\n",
    "    生成新的训练数据，用于分类器分类使用\n",
    "    \"\"\"\n",
    "    fn = (\"train.csv\") if train else (\"test.csv\")\n",
    "    fin = open(dpath + fn, 'r')\n",
    "    fout = open(dpath + \"RS_\" + fn, 'w')\n",
    "    \n",
    "    #忽略第一行（列名字）\n",
    "    fin.readline().strip().split(\",\")\n",
    "    \n",
    "    # write output header\n",
    "    if header:\n",
    "        ocolnames = [\"invited\", \"userCF_reco\", \"evtCF_reco\",\"svdCF_reco\",\"user_reco\", \"evt_p_reco\",\n",
    "        \"evt_c_reco\", \"user_pop\", \"frnd_infl\", \"evt_pop\"]\n",
    "        if train:\n",
    "            ocolnames.append(\"interested\")\n",
    "            ocolnames.append(\"not_interested\")\n",
    "        fout.write(\",\".join(ocolnames) + \"\\n\")\n",
    "    \n",
    "    ln = 0\n",
    "    for line in fin:\n",
    "        ln += 1\n",
    "        if ln%500 == 0:\n",
    "            print (\"%s:%d (userId, eventId)=(%s, %s)\" % (fn, ln, userId, eventId))\n",
    "            #break;\n",
    "      \n",
    "        cols = line.strip().split(\",\")\n",
    "        userId = cols[0]\n",
    "        eventId = cols[1]\n",
    "        invited = cols[2]\n",
    "      \n",
    "        userCF_reco = RS.userCFReco(userId, eventId)\n",
    "        itemCF_reco = RS.eventCFReco(userId, eventId)\n",
    "        svdCF_reco = RS.svdCFReco(userId, eventId)\n",
    "        \n",
    "        user_reco = RS.userReco(userId, eventId)\n",
    "        evt_p_reco, evt_c_reco = RS.eventReco(userId, eventId)\n",
    "        user_pop = RS.userPop(userId)\n",
    "     \n",
    "        frnd_infl = RS.friendInfluence(userId)\n",
    "        evt_pop = RS.eventPop(eventId)\n",
    "        ocols = [invited, userCF_reco, itemCF_reco, svdCF_reco,user_reco, evt_p_reco,\n",
    "        evt_c_reco, user_pop, frnd_infl, evt_pop]\n",
    "      \n",
    "        if train:\n",
    "            ocols.append(cols[4]) # interested\n",
    "            ocols.append(cols[5]) # not_interested\n",
    "        fout.write(\",\".join(map(lambda x: str(x), ocols)) + \"\\n\")\n",
    "    \n",
    "    fin.close()\n",
    "    fout.close()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 5.生成数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SVD Train...\n",
      "SVD training started\n",
      "SVD trained\n",
      "生成训练数据...\n",
      "\n",
      "train.csv:500 (userId, eventId)=(123290209, 1887085024)\n",
      "train.csv:1000 (userId, eventId)=(272886293, 199858305)\n",
      "train.csv:1500 (userId, eventId)=(395305791, 1582270949)\n",
      "train.csv:2000 (userId, eventId)=(527523423, 3272728211)\n",
      "train.csv:2500 (userId, eventId)=(651258472, 792632006)\n",
      "train.csv:3000 (userId, eventId)=(811791433, 524756826)\n",
      "train.csv:3500 (userId, eventId)=(985547042, 1269035551)\n",
      "train.csv:4000 (userId, eventId)=(1107615001, 173949238)\n",
      "train.csv:4500 (userId, eventId)=(1236336671, 3849306291)\n",
      "train.csv:5000 (userId, eventId)=(1414301782, 2652356640)\n",
      "train.csv:5500 (userId, eventId)=(1595465532, 955398943)\n",
      "train.csv:6000 (userId, eventId)=(1747091728, 2131379889)\n",
      "train.csv:6500 (userId, eventId)=(1914182220, 955398943)\n",
      "train.csv:7000 (userId, eventId)=(2071842684, 1076364848)\n",
      "train.csv:7500 (userId, eventId)=(2217853337, 3051438735)\n",
      "train.csv:8000 (userId, eventId)=(2338481531, 2525447278)\n",
      "train.csv:8500 (userId, eventId)=(2489551967, 520657921)\n",
      "train.csv:9000 (userId, eventId)=(2650493630, 87962584)\n",
      "train.csv:9500 (userId, eventId)=(2791418962, 4223848259)\n",
      "train.csv:10000 (userId, eventId)=(2903662804, 2791462807)\n",
      "train.csv:10500 (userId, eventId)=(3036141956, 3929507420)\n",
      "train.csv:11000 (userId, eventId)=(3176074542, 3459485614)\n",
      "train.csv:11500 (userId, eventId)=(3285425249, 2271782630)\n",
      "train.csv:12000 (userId, eventId)=(3410667855, 1063772489)\n",
      "train.csv:12500 (userId, eventId)=(3531604778, 2584839423)\n",
      "train.csv:13000 (userId, eventId)=(3686871863, 53495098)\n",
      "train.csv:13500 (userId, eventId)=(3833637800, 2415873572)\n",
      "train.csv:14000 (userId, eventId)=(3944021305, 2096772901)\n",
      "train.csv:14500 (userId, eventId)=(4075466480, 3567240505)\n",
      "train.csv:15000 (userId, eventId)=(4197193550, 1628057176)\n",
      "生成预测数据...\n",
      "\n",
      "test.csv:500 (userId, eventId)=(182290053, 2529072432)\n",
      "test.csv:1000 (userId, eventId)=(433510318, 4244463632)\n",
      "test.csv:1500 (userId, eventId)=(632808865, 2845303452)\n",
      "test.csv:2000 (userId, eventId)=(813611885, 2036538169)\n",
      "test.csv:2500 (userId, eventId)=(1010701404, 303459881)\n",
      "test.csv:3000 (userId, eventId)=(1210932037, 2529072432)\n",
      "test.csv:3500 (userId, eventId)=(1452921099, 2705317682)\n",
      "test.csv:4000 (userId, eventId)=(1623287180, 1626678328)\n",
      "test.csv:4500 (userId, eventId)=(1855201342, 2603032829)\n",
      "test.csv:5000 (userId, eventId)=(2083900381, 2529072432)\n",
      "test.csv:5500 (userId, eventId)=(2318415276, 2509151803)\n",
      "test.csv:6000 (userId, eventId)=(2528161539, 4025975316)\n",
      "test.csv:6500 (userId, eventId)=(2749110768, 4244406355)\n",
      "test.csv:7000 (userId, eventId)=(2927772127, 1532377761)\n",
      "test.csv:7500 (userId, eventId)=(3199685636, 1776393554)\n",
      "test.csv:8000 (userId, eventId)=(3393388475, 680270887)\n",
      "test.csv:8500 (userId, eventId)=(3601169721, 154434302)\n",
      "test.csv:9000 (userId, eventId)=(3828963415, 3067222491)\n",
      "test.csv:9500 (userId, eventId)=(4018723397, 2522610844)\n",
      "test.csv:10000 (userId, eventId)=(4180064266, 2658555390)\n"
     ]
    }
   ],
   "source": [
    "RS = RecommonderSystem()\n",
    "print (\"生成训练数据...\\n\")\n",
    "generateRSData(RS,train=True,  header=True)\n",
    "\n",
    "print (\"生成预测数据...\\n\")\n",
    "generateRSData(RS, train=False, header=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "基于物品的协同过滤的相似度计算部分感觉有些问题，望助教大大解答"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
